#include "SplitTool.h"

SplitTool::SplitTool()
{}

SplitTool::~SplitTool()
{
}

vector<WebPage> SplitTool::split_XML(const string& filename)
{
    XMLDocument doc;

    vector<WebPage> items;
    
    if(doc.LoadFile(filename.c_str()) != XML_SUCCESS)
    {
        cerr << "无法加载XML文件: " << filename << "\n";
        return items;
    }
    XMLElement *root = doc.RootElement();
    if(!root)
    {
        cerr << "无效的XML文件: 缺少根元素\n";
        return items;
    }
    XMLElement *channel = root->FirstChildElement("channel");
    if(!channel)
    {
        cerr << "无效的XML文件: 缺少channel元素\n";
        return items;
    }

    for (XMLElement *item = channel->FirstChildElement("item"); item; item = item->NextSiblingElement("item")) {
        WebPage rssItem;
        XMLElement *title = item->FirstChildElement("title");
        if (title && title->GetText()) {
            rssItem._title = title->GetText();
        }
        XMLElement *link = item->FirstChildElement("link");
        if (link && link->GetText()) {
            rssItem._link = link->GetText();
        }
        // XMLElement *pubDate = item->FirstChildElement("pubDate");
        // if (pubDate && pubDate->GetText()) {
        //     _pubDate = pubDate->GetText();
        // }
        // XMLElement *author = item->FirstChildElement("author");
        // if (author && author->GetText()) {
        //     rssItem.author = author->GetText();
        // }
        XMLElement *description = item->FirstChildElement("description");
        if (description && description->GetText()) {
            rssItem._content = description->GetText();
        }
        XMLElement *content= item->FirstChildElement("content");
        if (content && content->GetText()) {
            rssItem._content = content->GetText();
        }
        items.push_back(rssItem);
    }
    return items;
}

void SplitTool::save_to_JSON(vector<WebPage>& items, const string& filename)
{
    ofstream ofs(filename);
    if(!ofs.is_open())
    {
        cerr << "无法打开文件: " << filename << "\n";
        return;
    }

    for(size_t i = 0; i < items.size(); ++i)
    {
       const auto &item = items[i];
       ofs << "<doc>\n";
       ofs << "    <title>" << item._title << "</title>\n";
       ofs << "    <link>" << item._link << "</link>\n";
       ofs << "    <content>" << item._content<< "</content>\n";
       ofs << "</doc>\n";
    }
}

unordered_set<string> SplitTool::StopWords(const string & filename)
{
    unordered_set<string> stopWords;
    ifstream ifs;
    ifs.open(filename);
    if(!ifs.is_open())
    {
        cerr << "无法打开文件: " << filename << "\n";
    }

    string word;
    while(getline(ifs, word))
    {
        word.erase(0, word.find_first_not_of(" \t\n\r"));
        word.erase(word.find_last_not_of(" \t\n\r")+1);
        stopWords.insert(word);
    }

    return stopWords;
}

vector<string> SplitTool::tokenize(const string & text)
{
    vector<string> words;
    string word;
    for (char c : text)
    {
        if (isalnum(c))
        {
            char ch = std::tolower(static_cast<unsigned char>(c));
            word += ch;
        }
        else
        {
            if (!word.empty())
            {
                words.push_back(word);
                word.clear();
            }
        }
    }

    if(!word.empty())
    {
        words.push_back(word);
    }
    return words;
}


vector<string> SplitTool::cut(const string &filename)
{
    static unordered_set<string> cnStopWords = StopWords("En_yuliao/stop_words_eng.txt");
    vector<string> AllWords;
    ifstream file;
    file.open(filename);
    if (!file.is_open())
    {
        cerr << "无法打开文件：" << filename << "\n";
        return AllWords;
    }

    string line;
    while (getline(file, line))
    {
        vector<string> words = tokenize(line);

        vector<string> finalWords;
        //去掉通用词
        for(auto & word : words)
        {
            if(cnStopWords.find(word) == cnStopWords.end())
            {
                finalWords.push_back(word);
            }
        }

        //去掉停用词后再添加到结果中去
        AllWords.insert(AllWords.end(), finalWords.begin(), finalWords.end());
    }

    file.close();
    return AllWords;
}
